# -*- coding: utf-8 -*-
# @Time    : 2021/9/22 17:26
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: middlewares.py
# @Software: PyCharm

import random
import logging
import time
from scrapy.downloadermiddlewares.retry import RetryMiddleware, response_status_message
from scrapy_pyppeteer.request import PyppeteerRequest

logger = logging.getLogger(__name__)


class UserAgentMiddleware(object):
    def process_request(self, request, spider):
        user_agent = random.choice(spider.settings['USER_AGENTS'])
        request.headers['User-Agent'] = user_agent


class OuterNetProxyMiddleware(object):
    def process_request(self, request, spider):
        key_name, proxy_ip = spider.redis_conn.brpop('proxy', timeout=2 * 24 * 60 * 60)
        proxy_dict = {
            "http": "http://hifo:hifo1996@{}:65500".format(proxy_ip),
            "https": "https://hifo:hifo1996@{}:65500".format(proxy_ip)
        }
        request.meta['proxy'] = proxy_dict['http'] if request.url.startswith("http://") else proxy_dict['https']


class CustomRetryMiddleware(RetryMiddleware):
    def process_response(self, request, response, spider):
        if request.meta.get('dont_retry', False):
            return response
        if response.status in self.retry_http_codes:
            reason = response_status_message(response.status)
            return self._retry(request, reason, spider) or response
        if spider.name == 'beike':
            if 'loupan' in request.url:
                try:
                    check_str = response.xpath("//h1/text()").extract_first()
                    if isinstance(check_str, str):
                        assert ('人机' not in check_str)
                except:
                    retries = request.meta.get('retry_times', 0) + 1
                    logger.warning(
                        '{}:{} 被重定向到人机页面，更换代理ip尝试重试，第{}次重试'.format(spider.name, request.url, retries))
                    reason = response_status_message(response.status)
                    time.sleep(1.5)
                    return self._retry(request, reason, spider) or response
        if spider.name == 'anjuke':
            if 'captcha' in request.url:
                wait_for_str = 'div.key-list>div.item-mod' if '.html' in request.url else 'div.can-left div.des'
                retries = request.meta.get('retry_times', 0) + 1
                logger.warning(
                    '{}:{} 被重定向到验证码页面，更换代理ip尝试重试，第{}次重试'.format(spider.name, request.url, retries))
                reason = response_status_message(response.status)
                pyp_request = PyppeteerRequest(request.url, callback=request.callback, pretend=True,
                                               meta=request.meta,
                                               wait_for=wait_for_str,
                                               ignore_resource_types=['image', 'media'], )
                return self._retry(pyp_request, reason, spider) or response
        return response
